# -*- coding: utf-8 -*-
"""
Created on 2021-09-22 09:11:30
---------
@summary:
---------
@author: xubin
"""
from my_tool import my_tool as tool

import feapder
from feapder.db.mongodb import MongoDB

from my_tool.snowflake_id import IdWorker

from feapder.utils.tools import get_current_date, get_md5

from items.Huabio_item import HuabioItem

mongo_conn = MongoDB()
worker = IdWorker(1, 2, 0)


def insert_mongo(table, data):
    res = mongo_conn.add(table, data)
    print(res)


class HuabioSpider(feapder.AirSpider):
    def filter_none(self, data_list):
        return [_.strip() for _ in data_list if _.strip()]

    def get_url(self, nodes):
        for _ in nodes:
            title_list = _.xpath('.//li/a/text()').getall()
            value_list = _.xpath('.//@value').getall()
            res = list(map(lambda x, y: {x: y}, title_list, value_list))
            print(res)
            return res

    def get_other_url(self, nodes):
        for _ in nodes:
            title_list = _.xpath('.//li//label/text()').getall()
            value_list = _.xpath('.//input/@value').getall()
            res = list(map(lambda x, y: {x: y}, title_list, value_list))
            print(res)
            return res

    def process_url(self, res, base_url):
        url_list = []
        for item in res:
            for k, v in item.items():
                page_num = int("".join(list(filter(str.isdigit, k)))) // 10 + 2
                for p in range(1, page_num):
                    url = base_url.format(p, v)
                    url_list.append(url)

        return url_list

    def start_requests(self):
        yield feapder.Request("http://www.huabio.cn/product")

    def parse(self, request, response):
        # ul_antibody_type_nodes = response.xpath('//ul[@id="ulantibodyType"]')
        # res = self.get_url(ul_antibody_type_nodes)
        # base_url = 'http://www.huabio.cn/product?sort=asc&current={}&antibodyTypeId={}&tag=antibodyType'
        # url_list = self.process_url(res, base_url)
        # print(url_list)

        ####################################################################################################
        ul_applications_id_nodes = response.xpath('//ul[@id="ulapplications"]')
        res = self.get_other_url(ul_applications_id_nodes)
        base_url = 'http://www.huabio.cn/product?sort=asc&current={}&applicationsId={}&tag=applications'
        url_list = self.process_url(res, base_url)
        # print(url_list)

        ####################################################################################################
        # ul_species_id_nodes = response.xpath('//ul[@id="ulspecies"]')
        # res = self.get_other_url(ul_species_id_nodes)
        # base_url = 'http://www.huabio.cn/product?sort=asc&current={}&speciesId={}&tag=species'
        # url_list = self.process_url(res, base_url)
        # print(url_list)
        ####################################################################################################

        # ul_conjugate_id_nodes = response.xpath('//ul[@id="ulconjugate"]')
        # res = self.get_url(ul_conjugate_id_nodes)
        # base_url = 'http://www.huabio.cn/product?sort=asc&current={}&conjugateId={}&tag=conjugate'
        # url_list = self.process_url(res, base_url)
        # print(url_list)

        ####################################################################################################

        # base_url = 'http://www.huabio.cn/product?sort=asc&current={}&kokdvalidatedId={}&tag=kokdvalidated'
        # ul_conjugate_id_nodes = response.xpath('//ul[@id="ulkokdvalidated"]')
        # res = self.get_other_url(ul_conjugate_id_nodes)
        # url_list = self.process_url(res, base_url)
        # print(url_list)

        ####################################################################################################
        for url in url_list:
            yield feapder.Request(url=url, callback=self.second_parse)

    def second_parse(self, request, response):
        pro_list_nodes = response.xpath('//div[@class="pro_list_rg"]')
        for node in pro_list_nodes:
            item = HuabioItem()
            pro_name =''.join(self.filter_none(node.xpath('./div[@class="col-md-12 pro_item_"]/a/text()').getall())[0].split('\r\n'))
            pro_url = node.xpath('./div[@class="col-md-12 pro_item_"]/a/@href').getall()[0]
            col_md_nodes = response.xpath('.//div[@class="col-md-12"]')
            for i in col_md_nodes:
                res_node = i.xpath('.//div//text()').getall()
                if res_node[0] == '抗体类型:':
                    item.antibody_type = ''.join(res_node[1:])
                if res_node[0] == '货号:' or res_node[0] == 'Catalog No:':
                    item.accession = ''.join(res_node[1:])
                if res_node[0] == '应用:':
                    item.applications = ''.join(self.filter_none(res_node[1::]))
                if res_node[0] == '偶联物:':
                    item.conjugate = ''.join(res_node[1:])
                if res_node[0] == '反应物种:':
                    item.reactive_species = ''.join(self.filter_none(res_node[1:]))
                if res_node[0] == '产品类型:' or res_node[0] == 'Product type:':
                    item.product_type = ''.join(res_node[1:])
                if res_node[0] == '品牌:':
                    item.brand = ''.join(res_node[1:])
                if res_node[0] == '样本类型:':
                    item.sample_type = ''.join(res_node[1:])
                if res_node[0] == 'Tag:':
                    item.tag = ''.join(res_node[1:])
            item.id = worker.get_id()
            item.name = pro_name
            item.detail_url = pro_url

            item.source_code_id = get_md5(response.url)
            item.detail_source_code_id = get_md5(pro_url)
            item.create_by = 'xubin@%'
            item.update_by = 'xubin@%'
            item.create_on = get_current_date()
            item.update_on = get_current_date()
            item.batch_id = 4538223704501391361
            data_dict = {
                '_id': get_md5(response.url),
                'url': response.url,
                'createBy': 'xubin',
                'updateTime': get_current_date(),
                'html': tool.compress_content(response.text)
            }
            yield item
            yield feapder.Request(url=pro_url, callback=self.detail_page_save_to_mongo)
            yield insert_mongo(table='HuabioDetail', data=data_dict)

    def detail_page_save_to_mongo(self, request, response):
        data_dict = {
            '_id': get_md5(response.url),
            'url': response.url,
            'createBy': 'xubin',
            'updateTime': get_current_date(),
            'html': tool.compress_content(response.text)
        }

        yield insert_mongo(table='HuabioDetail', data=data_dict)


if __name__ == "__main__":
    HuabioSpider(thread_count=4).start()
